
# encoding=utf-8
# token类型定义：1-<,2->,3-<<,4->>,5-=>,6-普通字符串,7-空格或换行,8-::

# 解析代码文件为token序列
def read_tokens_from_file(file_name):
    content = open(file_name, 'r').read()
    content += ' '
    state = 0     # 状态
    strtmp = ''
    tokens = []
    currentline = 1
    for c in content:
        if c == '\n':
            currentline += 1
        if state == 0:
            if c == ' ' or c == '\r' or c == '\n' or c == '\t':
                state = 0
                strtmp += c
            elif c == '<':
                tokens.append({
                    'token_type': 7,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 1
                strtmp += c
            elif c == '>':
                tokens.append({
                    'token_type': 7,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 2
                strtmp += c
            elif c == '=':
                tokens.append({
                    'token_type': 7,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 3
                strtmp += c
            elif c == ':':
                tokens.append({
                    'token_type': 7,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 5
                strtmp += c
            else:
                tokens.append({
                    'token_type': 7,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 4
                strtmp += c
        elif state == 1:
            if c == ' ' or c == '\r' or c == '\n' or c == '\t':
                tokens.append({
                    'token_type': 1,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
                strtmp += c
            elif c == '<':
                strtmp += c
                tokens.append({
                    'token_type': 3,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
            elif c == '>':
                tokens.append({
                    'token_type': 1,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 2
                strtmp += c
            elif c == '=':
                tokens.append({
                    'token_type': 1,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 3
                strtmp += c
            elif c == ':':
                tokens.append({
                    'token_type': 1,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 5
                strtmp += c
            else:
                tokens.append({
                    'token_type': 1,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 4
                strtmp += c
        elif state == 2:
            if c == ' ' or c == '\r' or c == '\n' or c == '\t':
                tokens.append({
                    'token_type': 2,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
                strtmp += c
            elif c == '<':
                tokens.append({
                    'token_type': 2,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 1
                strtmp += c
            elif c == '>':
                strtmp += c
                tokens.append({
                    'token_type': 4,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
            elif c == '=':
                tokens.append({
                    'token_type': 2,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 3
                strtmp += c
            elif c == ':':
                tokens.append({
                    'token_type': 2,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 5
                strtmp += c
            else:
                tokens.append({
                    'token_type': 2,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 4
                strtmp += c
        elif state == 3:
            if c == ' ' or c == '\r' or c == '\n' or c == '\t':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
                strtmp += c
            elif c == '<':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 1
                strtmp += c
            elif c == '>':
                strtmp += c
                tokens.append({
                    'token_type': 5,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
            elif c == '|':
                strtmp += c
                tokens.append({
                    'token_type': 9,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
            elif c == '=':
                state = 4
                strtmp += c
            elif c == ':':
                state = 4
                strtmp += c
            else:
                state = 4
                strtmp += c
        elif state == 4:
            if c == ' ' or c == '\r' or c == '\n' or c == '\t':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
                strtmp += c
            elif c == '<':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 1
                strtmp += c
            elif c == '>':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 2
                strtmp += c
            elif c == '=':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 3
                strtmp += c
            elif c == ':':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 5
                strtmp += c
            else:
                state = 4
                strtmp += c
        elif state == 5:
            if c == ' ' or c == '\r' or c == '\n' or c == '\t':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
                strtmp += c
            elif c == '<':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 1
                strtmp += c
            elif c == '>':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 2
                strtmp += c
            elif c == '=':
                tokens.append({
                    'token_type': 6,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 3
                strtmp += c
            elif c == ':':
                strtmp += c
                tokens.append({
                    'token_type': 8,
                    'content': strtmp,
                    'line': currentline
                })
                strtmp = ''
                state = 0
            else:
                state = 4
                strtmp += c
    # 去掉无效token
    tlen = len(tokens)
    for i in range(tlen - 1, -1, -1):
        if tokens[i]['content'] == "":
            tokens.pop(i)

    return tokens



